{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/getlimits.py:499: UserWarning: The value of the smallest subnormal for <class 'numpy.float64'> type is zero.\n",
      "  setattr(self, word, getattr(machar, word).flat[0])\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for <class 'numpy.float64'> type is zero.\n",
      "  return self._float_to_str(self.smallest_subnormal)\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/getlimits.py:499: UserWarning: The value of the smallest subnormal for <class 'numpy.float32'> type is zero.\n",
      "  setattr(self, word, getattr(machar, word).flat[0])\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for <class 'numpy.float32'> type is zero.\n",
      "  return self._float_to_str(self.smallest_subnormal)\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 1.300 seconds.\n",
      "Prefix dict has been built successfully.\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/Cython/Compiler/Main.py:384: FutureWarning: Cython directive 'language_level' not set, using '3str' for now (Py3). This has changed from earlier releases! File: /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/mindnlp/transformers/models/graphormer/algos_graphormer.pyx\n",
      "  tree = Parsing.p_module(s, pxd, full_module_name)\n",
      "In file included from /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/include/numpy/ndarraytypes.h:1960:0,\n",
      "                 from /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n",
      "                 from /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/include/numpy/arrayobject.h:5,\n",
      "                 from /home/ma-user/.pyxbld/temp.linux-aarch64-cpython-39/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/mindnlp/transformers/models/graphormer/algos_graphormer.c:1167:\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n",
      " #warning \"Using deprecated NumPy API, disable it with \" \\\n",
      "  ^~~~~~~\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import json\n",
    "\n",
    "import mindspore\n",
    "from mindspore.dataset import transforms\n",
    "from mindspore import nn\n",
    "\n",
    "from mindnlp.dataset import load_dataset\n",
    "\n",
    "from mindnlp.engine import Trainer, Evaluator\n",
    "from mindnlp.engine.callbacks import CheckpointCallback, BestModelCallback\n",
    "from mindnlp.metrics import Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mindspore.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-12-24 12:58:26--  https://raw.githubusercontent.com/ids-cv/wrime/master/wrime-ver2.tsv\n",
      "Resolving proxy-notebook.modelarts.com (proxy-notebook.modelarts.com)... 192.168.0.33\n",
      "Connecting to proxy-notebook.modelarts.com (proxy-notebook.modelarts.com)|192.168.0.33|:8083... connected.\n",
      "Proxy request sent, awaiting response... 200 OK\n",
      "Length: 8182156 (7.8M) [text/plain]\n",
      "Saving to: ‘./wrime_data/wrime-ver2.tsv’\n",
      "\n",
      "wrime-ver2.tsv      100%[===================>]   7.80M   718KB/s    in 11s     \n",
      "\n",
      "2024-12-24 12:58:37 (748 KB/s) - ‘./wrime_data/wrime-ver2.tsv’ saved [8182156/8182156]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 下载tsv格式wrime数据集\n",
    "!mkdir -p ./wrime_data\n",
    "!wget -P ./wrime_data https://raw.githubusercontent.com/ids-cv/wrime/master/wrime-ver2.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved train data to ./wrime_data/train.json, size = 20149\n",
      "Saved validation data to ./wrime_data/val.json, size = 1608\n",
      "Saved test data to ./wrime_data/test.json, size = 1781\n"
     ]
    }
   ],
   "source": [
    "# 将TSV数据集文件进行划分并转换为JSON文件\n",
    "\n",
    "# 定义保存路径\n",
    "_OUTPUT_DIR = \"./wrime_data\"  # 保存JSON文件的目录\n",
    "\n",
    "# 定义输出文件名\n",
    "TRAIN_JSON = f\"{_OUTPUT_DIR}/train.json\"\n",
    "VAL_JSON = f\"{_OUTPUT_DIR}/val.json\"\n",
    "TEST_JSON = f\"{_OUTPUT_DIR}/test.json\"\n",
    "\n",
    "\n",
    "# 将TSV文件转换为JSON文件\n",
    "def tsv_to_json(tsv_file, train_json, val_json, test_json, remove_neutral=True):\n",
    "    train_data = []\n",
    "    val_data = []\n",
    "    test_data = []\n",
    "\n",
    "    with open(tsv_file, 'r', encoding='utf-8') as f:\n",
    "        reader = csv.DictReader(f, delimiter='\\t')\n",
    "        for row in reader:\n",
    "            sentiment_score = int(row[\"Avg. Readers_Sentiment\"])\n",
    "            label = None\n",
    "\n",
    "            if sentiment_score > 0:\n",
    "                label = \"positive\"\n",
    "                label = 0\n",
    "            elif sentiment_score < 0:\n",
    "                label = \"negative\"\n",
    "                label = 1\n",
    "            else:\n",
    "                label = \"neutral\"\n",
    "                label = 2\n",
    "\n",
    "            if remove_neutral and label == 2:\n",
    "                continue\n",
    "\n",
    "            data_point = {\n",
    "                \"sentence\": row[\"Sentence\"],\n",
    "                \"label\": label,\n",
    "                # \"user_id\": int(row[\"UserID\"]),\n",
    "                # \"datetime\": row[\"Datetime\"].strip()\n",
    "            }\n",
    "\n",
    "            # 根据 Train/Dev/Test 列划分数据集\n",
    "            if row[\"Train/Dev/Test\"].lower() == \"train\":\n",
    "                train_data.append(data_point)\n",
    "            elif row[\"Train/Dev/Test\"].lower() == \"dev\":\n",
    "                val_data.append(data_point)\n",
    "            elif row[\"Train/Dev/Test\"].lower() == \"test\":\n",
    "                test_data.append(data_point)\n",
    "\n",
    "\n",
    "    # 保存为JSON文件\n",
    "    with open(train_json, 'w', encoding='utf-8') as f:\n",
    "        json.dump(train_data, f, ensure_ascii=False, indent=4)\n",
    "        print(f\"Saved train data to {train_json}, size = {len(train_data)}\")\n",
    "\n",
    "    with open(val_json, 'w', encoding='utf-8') as f:\n",
    "        json.dump(val_data, f, ensure_ascii=False, indent=4)\n",
    "        print(f\"Saved validation data to {val_json}, size = {len(val_data)}\")\n",
    "\n",
    "    with open(test_json, 'w', encoding='utf-8') as f:\n",
    "        json.dump(test_data, f, ensure_ascii=False, indent=4)\n",
    "        print(f\"Saved test data to {test_json}, size = {len(test_data)}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # 将TSV文件转换为JSON文件\n",
    "    tsv_file = f\"{_OUTPUT_DIR}/wrime-ver2.tsv\"\n",
    "    tsv_to_json(tsv_file, TRAIN_JSON, VAL_JSON, TEST_JSON, remove_neutral=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating train split: 20149 examples [00:00, 138648.14 examples/s]\n",
      "Generating validation split: 1608 examples [00:00, 91531.96 examples/s]\n"
     ]
    }
   ],
   "source": [
    "# 加载训练集和验证集\n",
    "wrime_train = load_dataset(\"json\", data_files={\"train\": \"./wrime_data/train.json\"})\n",
    "wrime_val = load_dataset(\"json\", data_files={\"validation\": \"./wrime_data/val.json\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset column: ['sentence', 'label']\n",
      "dataset size: 20149\n",
      "dataset batch size: 1\n",
      "\n",
      "dataset column: ['sentence', 'label']\n",
      "dataset size: 1608\n",
      "dataset batch size: 1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def show_dataset_info(dataset):\n",
    "    print(\"dataset column: {}\".format(dataset.get_col_names()))\n",
    "    print(\"dataset size: {}\".format(dataset.get_dataset_size()))\n",
    "    print(\"dataset batch size: {}\\n\".format(dataset.get_batch_size()))\n",
    "show_dataset_info(wrime_train)\n",
    "show_dataset_info(wrime_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])\n",
      "dict_values([[2, 12575, 3], [0, 0, 0], [1, 1, 1]])\n"
     ]
    }
   ],
   "source": [
    "from mindnlp.transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained('tohoku-nlp/bert-base-japanese-v3')\n",
    "test_tokenized = tokenizer('世界')\n",
    "print(test_tokenized.keys())\n",
    "print(test_tokenized.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def process_dataset(dataset, tokenizer, max_seq_len=256, batch_size=32, shuffle=False):\n",
    "\n",
    "    is_ascend = mindspore.get_context('device_target') == 'Ascend'\n",
    "    def tokenize(text):\n",
    "        if is_ascend:\n",
    "            tokenized = tokenizer(text, padding='max_length', truncation=True, max_length=max_seq_len)\n",
    "        else:\n",
    "            tokenized = tokenizer(text, truncation=True, max_length=max_seq_len)\n",
    "        return tokenized['input_ids'], tokenized['token_type_ids'], tokenized['attention_mask']\n",
    "\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(batch_size)\n",
    "\n",
    "    # map dataset\n",
    "    dataset = dataset.map(operations=[tokenize], input_columns=\"sentence\", output_columns=['input_ids', 'token_type_ids', 'attention_mask'])   \n",
    "    dataset = dataset.map(operations=transforms.TypeCast(mindspore.int32), input_columns=\"label\", output_columns=\"labels\")\n",
    "    # batch dataset\n",
    "    if is_ascend:\n",
    "        dataset = dataset.batch(batch_size)\n",
    "    else:\n",
    "        dataset = dataset.padded_batch(batch_size, pad_info={'input_ids': (None, tokenizer.pad_token_id),\n",
    "                                                             'token_type_ids': (None, 0),\n",
    "                                                             'attention_mask': (None, 0)})\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train = process_dataset(wrime_train, tokenizer, shuffle=True)\n",
    "dataset_val = process_dataset(wrime_val, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "前 10 个学习率: [1.0582010582010582e-07, 2.1164021164021165e-07, 3.174603174603175e-07, 4.232804232804233e-07, 5.291005291005291e-07, 6.34920634920635e-07, 7.407407407407407e-07, 8.465608465608466e-07, 9.523809523809525e-07, 1.0582010582010582e-06]\n",
      "最后 10 个学习率: [1.0582010582010694e-07, 9.406231628453776e-08, 8.230452674897083e-08, 7.054673721340388e-08, 5.8788947677836936e-08, 4.703115814226999e-08, 3.527336860670305e-08, 2.3515579071133888e-08, 1.1757789535566944e-08, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# 动态学习率计算\n",
    "def dynamic_lr(lr, total_step, warmup_ratio=0.1):\n",
    "    \"\"\"\n",
    "    计算线性学习率调度器的学习率列表。\n",
    "\n",
    "    参数:\n",
    "    lr (float): 初始学习率\n",
    "    total_step (int): 总的训练步骤数\n",
    "    warmup_ratio (float): warmup 阶段的比例，默认为 0.1\n",
    "\n",
    "    返回:\n",
    "    lrs (list): 学习率列表\n",
    "    \"\"\"\n",
    "    lrs = []\n",
    "    warmup_steps = int(total_step * warmup_ratio)  # 计算 warmup 步数\n",
    "    decay_steps = total_step - warmup_steps  # 计算衰减阶段的步数\n",
    "\n",
    "    # Warmup 阶段：学习率从 0 线性增加到初始学习率\n",
    "    for i in range(1, warmup_steps+1):\n",
    "        warmup_factor = float(i) / max(1, warmup_steps)\n",
    "        lrs.append(lr * warmup_factor)\n",
    "\n",
    "    # 衰减阶段：学习率从初始学习率线性衰减到 0\n",
    "    for i in range(1, decay_steps+1):\n",
    "        decay_factor = 1.0 - float(i) / max(1, decay_steps)\n",
    "        lrs.append(lr * decay_factor)\n",
    "\n",
    "    return lrs\n",
    "\n",
    "# 数据集大小\n",
    "dataset_train_size = len(dataset_train)  # 假设有 1000 个样本\n",
    "total_step = 3 * dataset_train_size  # 3 个 epoch 的总步数\n",
    "\n",
    "# 定义初始学习率和 warmup 比例\n",
    "lr = 2e-5\n",
    "warmup_ratio = 0.1\n",
    "\n",
    "# 生成学习率列表\n",
    "decay_lr = dynamic_lr(lr=lr, total_step=total_step, warmup_ratio=warmup_ratio)\n",
    "\n",
    "# 打印前 10 个学习率\n",
    "print(\"前 10 个学习率:\", decay_lr[:10])\n",
    "# 打印最后 10 个学习率\n",
    "print(\"最后 10 个学习率:\", decay_lr[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following parameters in checkpoint files are not loaded:\n",
      "['bert.embeddings.position_ids', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n"
     ]
    }
   ],
   "source": [
    "from mindnlp.transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained('tohoku-nlp/bert-base-japanese-v3', num_labels=2)\n",
    "\n",
    "optimizer = nn.AdamWeightDecay(model.trainable_params(), learning_rate=decay_lr)\n",
    "\n",
    "\n",
    "metric = Accuracy()\n",
    "\n",
    "ckpoint_cb = CheckpointCallback(save_path='checkpoint', ckpt_name='bert_japanese_wrime_finetune', epochs=1, keep_checkpoint_max=2)\n",
    "best_model_cb = BestModelCallback(save_path='checkpoint', ckpt_name='bert_japanese_wrime_finetune_best', auto_load=True)\n",
    "\n",
    "trainer = Trainer(network=model, train_dataset=dataset_train,\n",
    "                  eval_dataset=dataset_val, metrics=metric,\n",
    "                  epochs=3, optimizer=optimizer, callbacks=[ckpoint_cb, best_model_cb],\n",
    "                  jit=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The train will start from the checkpoint saved in 'checkpoint'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/630 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 100%|█████████▉| 629/630 [03:45<00:00,  3.15it/s, loss=0.27174908]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 100%|██████████| 630/630 [03:54<00:00,  2.68it/s, loss=0.27164856]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint: 'bert_japanese_wrime_finetune_epoch_0.ckpt' has been saved in epoch: 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluate: 100%|██████████| 51/51 [00:07<00:00,  6.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9185323383084577}\n",
      "---------------Best Model: 'bert_japanese_wrime_finetune_best.ckpt' has been saved in epoch: 0.---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 630/630 [03:36<00:00,  2.91it/s, loss=0.1069226]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint: 'bert_japanese_wrime_finetune_epoch_1.ckpt' has been saved in epoch: 1.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluate: 100%|██████████| 51/51 [00:07<00:00,  6.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9203980099502488}\n",
      "---------------Best Model: 'bert_japanese_wrime_finetune_best.ckpt' has been saved in epoch: 1.---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 630/630 [03:34<00:00,  2.94it/s, loss=0.045812417]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum number of stored checkpoints has been reached.\n",
      "Checkpoint: 'bert_japanese_wrime_finetune_epoch_2.ckpt' has been saved in epoch: 2.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluate: 100%|██████████| 51/51 [00:07<00:00,  7.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9409203980099502}\n",
      "---------------Best Model: 'bert_japanese_wrime_finetune_best.ckpt' has been saved in epoch: 2.---------------\n",
      "Loading best model from 'checkpoint' with '['Accuracy']': [0.9409203980099502]...\n",
      "---------------The model is already load the best model from 'bert_japanese_wrime_finetune_best.ckpt'.---------------\n"
     ]
    }
   ],
   "source": [
    "trainer.run(tgt_columns=\"labels\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluate: 100%|██████████| 51/51 [00:06<00:00,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9409203980099502}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluator = Evaluator(network=model, eval_dataset=dataset_val, metrics=metric)\n",
    "evaluator.run(tgt_columns=\"labels\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
